{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 确认URL数量"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# 清理文件名\n",
    "from utils import listdir, simple_clean_text\n",
    "\n",
    "allpath = []\n",
    "listdir('/home/d/papers_data/', allpath)\n",
    "len(allpath)\n",
    "\n",
    "import os\n",
    "\n",
    "for path in allpath:\n",
    "    pdf_name = path.split('/')[-1].split('.pdf')[0]\n",
    "    new_pdf_name = simple_clean_text(pdf_name)\n",
    "    new_path = path.replace(pdf_name, new_pdf_name)\n",
    "    os.rename(path, new_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# 查找遗漏pdf文件\n",
    "from utils import get_json_file_paths\n",
    "import json\n",
    "\n",
    "# 从dict构建path\n",
    "json_paths = get_json_file_paths('./url_json/')\n",
    "json_paths[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./url_json\\\\AAAI_2014_2015_2016_2017_2018_2019.json'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 直接查找json文件\n",
    "from utils import listdir\n",
    "import json\n",
    "\n",
    "json_paths = []\n",
    "listdir('./url_json', json_paths, ['.json'])\n",
    "json_paths[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AAAI_2014 : 474\n",
      "AAAI_2015 : 672\n",
      "AAAI_2016 : 691\n",
      "AAAI_2017 : 785\n",
      "AAAI_2018 : 1100\n",
      "AAAI_2019 : 1341\n",
      "ACL_2014 : 328\n",
      "ACL_2015 : 358\n",
      "ACL_2016 : 379\n",
      "ACL_2017 : 352\n",
      "ACL_2018 : 653\n",
      "ACL_2019 : 1366\n",
      "EMNLP_2014 : 226\n",
      "EMNLP_2015 : 312\n",
      "EMNLP_2016 : 264\n",
      "EMNLP_2017 : 350\n",
      "EMNLP_2018 : 999\n",
      "EMNLP_2019 : 1094\n",
      "ICLR_2017_MAIN : 490\n",
      "ICLR_2018_ORAL : 990\n",
      "ICLR_2019_ORAL : 24\n",
      "ICLR_2019_POSTER : 478\n",
      "ICLR_2020_POSTER : 532\n",
      "ICLR_2020_SPOTLIGHT : 107\n",
      "ICLR_2020_ORAL : 48\n",
      "ICML_2017 : 433\n",
      "ICML_2018 : 621\n",
      "IJCAI_2015 : 649\n",
      "IJCAI_2016 : 651\n",
      "IJCAI_2017 : 777\n",
      "IJCAI_2018 : 867\n",
      "IJCAI_2019 : 964\n",
      "KDD_2017 : 217\n",
      "KDD_2018 : 292\n",
      "KDD_2019 : 321\n",
      "NAACL_2015 : 239\n",
      "NAACL_2016 : 224\n",
      "NAACL_2018 : 569\n",
      "NAACL_2019 : 729\n",
      "NIPS_2017 : 679\n",
      "NIPS_2018 : 1008\n",
      "NIPS_2019 : 1427\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "25080"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count = 0\n",
    "for json_path in json_paths:\n",
    "    org = json_path.split('/')[-1].split('_')[0]\n",
    "    with open(json_path, 'rb') as f:\n",
    "        j = json.load(f)\n",
    "    for key in j.keys():\n",
    "        print(f'{key} : {len(j[key])}')\n",
    "        count += len(j[key])\n",
    "count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../../papers_data/ICLR/ICLR_2018_ORAL/Binarized Back-Propagation Training Binarized Neural Networks with Binarized Gradients.txt\n",
      "https://openreview.net/pdf?id=ryKRRsm0Z\n",
      "../../papers_data/KDD/KDD_2019/TV Advertisement Scheduling by Learning Expert Intentions.txt\n",
      "https://dl.acm.org/doi/pdf/10.1145/3292500.3330768?download=true\n",
      "../../papers_data/KDD/KDD_2019/Whole Page Optimization with Global Constraints.txt\n",
      "https://dl.acm.org/doi/pdf/10.1145/3292500.3330675?download=true\n"
     ]
    }
   ],
   "source": [
    "# 检测dict对应的txt是否完整\n",
    "import re, os\n",
    "from utils import listdir, simple_clean_text\n",
    "base_url = r'../../papers_data'\n",
    "\n",
    "for json_path in json_paths:\n",
    "    org = json_path.split('/')[-1].split('_')[0]\n",
    "    with open(json_path, 'r', encoding='utf8') as f:\n",
    "        j = json.load(f)\n",
    "    for org_year in j.keys():\n",
    "        org = org_year.split('_')[0]\n",
    "        for _name in j[org_year]:\n",
    "            name = simple_clean_text(_name)\n",
    "            path = f'{base_url}/{org}/{org_year}/{name}.txt'\n",
    "            if not os.path.exists(path): print(f'{path}\\n{j[org_year][_name]}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cant_dw_list = ['Binarized Back-Propagation Training Binarized Neural Networks with Binarized Gradients.pdf',\n",
    "               'TV Advertisement Scheduling by Learning Expert Intentions.pdf',\n",
    "               'Whole Page Optimization with Global Constraints.pdf']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PDF完整性检查\n",
    "\n",
    "- AAAI\n",
    "\n",
    "存在72篇未能下载,原因是url可能要替换为`IAAI`或者`EAAI`.使用`handle_by_hand_AAAI.py`处理.处理后\n",
    "\n",
    "- ICLR\n",
    "\n",
    "共1条. 无法下载,谷歌搜不到.\n",
    "\n",
    "```json\n",
    "ERROR AFTER RETRY:['ICLR_2018_ORAL', 'Binarized Back-Propagation: Training Binarized Neural Networks with Binarized Gradients', 'https://openreview.net/pdf?id=ryKRRsm0Z']\n",
    "```\n",
    "\n",
    "\n",
    "- KDD\n",
    "\n",
    "共14条.作者未公开. 手动搜索谷歌学术和arxiv,只剩2条未找到.\n",
    "\n",
    "```json\n",
    "ERROR AFTER RETRY:['KDD_2019', 'TV Advertisement Scheduling by Learning Expert Intentions', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330768?download=true']\n",
    "ERROR AFTER RETRY:['KDD_2019', 'Topic-Enhanced Memory Networks for Personalised Point-of-Interest Recommendation', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330781?download=true']\n",
    "ERROR AFTER RETRY:['KDD_2019', 'Towards Identifying Impacted Users in Cellular Services', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330711?download=true']\n",
    "ERROR AFTER RETRY:['KDD_2019', 'Towards Knowledge-Based Personalized Product Description Generation in E-commerce', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330725?download=true']\n",
    "ERROR AFTER RETRY:['KDD_2019', 'Towards sustainable dairy management - a machine learning enhanced method for estrus detection', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330712?download=true']\n",
    "ERROR AFTER RETRY:['KDD_2019', 'TrajGuard: A Comprehensive Trajectory Copyright Protection Scheme', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330685?download=true']\n",
    "ERROR AFTER RETRY:['KDD_2019', 'Two-Sided Fairness for Repeated Matchings in Two-Sided Markets: A Case Study of a Ride-Hailing Platform', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330793?download=true']\n",
    "ERROR AFTER RETRY:['KDD_2019', 'Uncovering the Co-driven Mechanism of Social and Content Links in User Churn Phenomena', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330736?download=true']\n",
    "ERROR AFTER RETRY:['KDD_2019', 'Understanding Consumer Journey using Attention based Recurrent Neural Networks', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330753?download=true']\n",
    "ERROR AFTER RETRY:['KDD_2019', 'Understanding the Role of Style in E-commerce Shopping', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330760?download=true']\n",
    "ERROR AFTER RETRY:['KDD_2019', 'Unsupervised Clinical Language Translation', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330710?download=true']\n",
    "ERROR AFTER RETRY:['KDD_2019', 'UrbanFM: Inferring Fine-Grained Urban Flows', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330646?download=true']\n",
    "ERROR AFTER RETRY:['KDD_2019', 'Using Twitter to Predict When Vulnerabilities will be Exploited', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330742?download=true']\n",
    "ERROR AFTER RETRY:['KDD_2019', 'Whole Page Optimization with Global Constraints', 'https://dl.acm.org/doi/pdf/10.1145/3292500.3330675?download=true']\n",
    "```\n",
    "\n",
    "- NIPS\n",
    "\n",
    "网页中标签<a>下存在别的标签,导致获取文件名不完整. `handle_by_hand_NIPS.py`解决."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "!python build_bulk_for_es.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# bulk错误处理\n",
    "\n",
    "自动copy相应pdf并生成txt文件,人工填完后自动读取,建立索引"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re, os, shutil\n",
    "\n",
    "with open('./pdf_es_json/timeout_err.txt', 'r', encoding='utf8') as f:\n",
    "    t = f.read()\n",
    "patten = re.compile(r'/home/d/(.*?).pdf')\n",
    "res1 = patten.findall(t)\n",
    "len(res1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'papers_data/ACL/ACL_2018/Do Neural Network Cross-Modal Mappings Really Bridge Modalities'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res1[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for path in res1:\n",
    "    folder, org, org_year, name = path.split('/')\n",
    "    pdf_name = name + '.pdf'\n",
    "    if pdf_name in cant_dw_list: continue\n",
    "    new_foolder = folder + '_by_hand'\n",
    "    \n",
    "    if not os.path.exists(f'{new_foolder}'): os.makedirs(f'{new_foolder}')\n",
    "    \n",
    "    _file_from = f'/home/d/{folder}/{org}/{org_year}/{pdf_name}'\n",
    "    _file_to = f'{new_foolder}/{pdf_name}'\n",
    "    shutil.copy(_file_from, _file_to)\n",
    "    txt_name = name + '.txt'\n",
    "    with open(f'{new_foolder}/{txt_name}', 'w') as f:\n",
    "        f.write('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 确认ES中数据的完整性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json, requests\n",
    "from utils import simple_clean_text, get_json_file_paths\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "def exists_query(org, name):\n",
    "    data = {\n",
    "            \"_source\": {\n",
    "            \"include\": [\n",
    "                \"Name\",\n",
    "                ]\n",
    "                },\n",
    "      \"query\": {\n",
    "        \"bool\": {\n",
    "          \"must\": [\n",
    "            { \"match_phrase\": { \"Name\": name } }\n",
    "          ],\n",
    "        }\n",
    "      }\n",
    "    }\n",
    "    headers = {'Content-Type': 'application/json'}\n",
    "\n",
    "    r = requests.get(f\"http://localhost:9200/{org}/_search\", headers = headers, data = json.dumps(data))\n",
    "    res_json = json.loads(r.text)\n",
    "    total_value = res_json['hits']['total']['value']\n",
    "    return int(total_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe8eeb3382734204aeb21c7ec7b7e67b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='ACL_2014', layout=Layout(flex='2'), max=328, style=ProgressSt…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0edefb30dd3245f9a3a5b38c2549c05b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='ACL_2015', layout=Layout(flex='2'), max=358, style=ProgressSt…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ecdbe13622642cf902b2e96f0806bac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='ACL_2016', layout=Layout(flex='2'), max=379, style=ProgressSt…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8611023e6a494b65990349c61401e48e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='ACL_2017', layout=Layout(flex='2'), max=352, style=ProgressSt…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "471029386d75447fa6a8ca2c5644df74",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='ACL_2018', layout=Layout(flex='2'), max=653, style=ProgressSt…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c5453bf181d43d8a6e0d0e8be0458ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='ACL_2019', layout=Layout(flex='2'), max=1366, style=ProgressS…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8d8936be57741f6ac33c0bac2d53237",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='AAAI_2014', layout=Layout(flex='2'), max=474, style=ProgressS…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf21c2137e2442e2b4ae21403e74be7a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='AAAI_2015', layout=Layout(flex='2'), max=672, style=ProgressS…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c07b5d6b0f6f4b508286a0c8db85b56a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='AAAI_2016', layout=Layout(flex='2'), max=691, style=ProgressS…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "206c801669974e4cb791bc2b4ecb61b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='AAAI_2017', layout=Layout(flex='2'), max=785, style=ProgressS…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14aebbc67ac44daf94cab3e9a3b7f017",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='AAAI_2018', layout=Layout(flex='2'), max=1100, style=Progress…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18fa3739cc794443a3068ab0654820d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='AAAI_2019', layout=Layout(flex='2'), max=1341, style=Progress…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2be3381e9a5d474898434095f394c892",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='EMNLP_2014', layout=Layout(flex='2'), max=226, style=Progress…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "053c3207c2b74f7d995ef6a6337f48bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='EMNLP_2015', layout=Layout(flex='2'), max=312, style=Progress…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51b3b906d0aa475b8dbaf4bf4c34351e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='EMNLP_2016', layout=Layout(flex='2'), max=264, style=Progress…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e45726df03c2474f9d86034ac8e7a7d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='EMNLP_2017', layout=Layout(flex='2'), max=350, style=Progress…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca57ca4369274f608ead75597859eee8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='EMNLP_2018', layout=Layout(flex='2'), max=999, style=Progress…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78c592017a7449758c085f34a7bbdcf7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='EMNLP_2019', layout=Layout(flex='2'), max=1094, style=Progres…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f82e3338a06d4e10a42ab940c8f332e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='ICLR_2017_MAIN', layout=Layout(flex='2'), max=490, style=Prog…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ce27dd2189340b880a03621a207d346",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='ICLR_2018_ORAL', layout=Layout(flex='2'), max=990, style=Prog…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0: ICLR: Binarized Back-Propagation Training Binarized Neural Networks with Binarized Gradients\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46f07482cff243c3a92de29c47b6e636",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='ICLR_2019_ORAL', layout=Layout(flex='2'), max=24, style=Progr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8fdf0cbf46744c84aca9e77f7ff71124",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='ICLR_2019_POSTER', layout=Layout(flex='2'), max=478, style=Pr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad8f9c4c5f6e436fbcbddb212ed13de3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='ICML_2017', layout=Layout(flex='2'), max=433, style=ProgressS…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c93dcd1d6e97421685d723e07a378e1e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='ICML_2018', layout=Layout(flex='2'), max=621, style=ProgressS…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "17f882a453394bd990bfc50ab3edc9f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='KDD_2017', layout=Layout(flex='2'), max=217, style=ProgressSt…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e79d9c5dc03940f48b204115c9c94b3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='KDD_2018', layout=Layout(flex='2'), max=292, style=ProgressSt…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10a5138cccd54fee8d6e6a657589bd7a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='KDD_2019', layout=Layout(flex='2'), max=321, style=ProgressSt…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0: KDD: TV Advertisement Scheduling by Learning Expert Intentions\n",
      "0: KDD: Whole Page Optimization with Global Constraints\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dabea2a0daf64a93a705ca721856a528",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='NIPS_2017', layout=Layout(flex='2'), max=679, style=ProgressS…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe19ee70fe60422db27d89f4d3b89997",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='NIPS_2018', layout=Layout(flex='2'), max=1008, style=Progress…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16e3b3c5a3a14a1a8b432b28c1bd234e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='NIPS_2019', layout=Layout(flex='2'), max=1427, style=Progress…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "json_paths = get_json_file_paths('./url_json/')\n",
    "\n",
    "for json_path in json_paths:\n",
    "    org = json_path.split('/')[-1].split('_')[0]\n",
    "    with open(json_path, 'rb') as f:\n",
    "        j = json.load(f)\n",
    "    for org_year in j.keys():\n",
    "        org = org_year.split('_')[0]\n",
    "        for name in tqdm(j[org_year], ncols = 500, desc = org_year):\n",
    "            url = j[org_year][name]\n",
    "            name = simple_clean_text(name)\n",
    "            total_value = exists_query(org = org.lower(), name = name)\n",
    "            if total_value == 0: print(f'0: {org}: {name}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 直接查找json文件\n",
    "from utils import listdir\n",
    "import json\n",
    "\n",
    "json_paths = []\n",
    "listdir('./url_json', json_paths, ['.json'])\n",
    "json_paths[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'2014', '2015'}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xx = ['2014', '2014', '2015']\n",
    "import collections\n",
    "\n",
    "oo = collections.defaultdict(set)\n",
    "for x in xx:\n",
    "    oo['ss'].add(x)\n",
    "\n",
    "# oo['ss'] = sorted(set(oo['ss']))\n",
    "oo['ss']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2, 4, 5, 6]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q = set([5 ,6,4, 2, 6])\n",
    "q = sorted(q)\n",
    "q"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
